{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### Install the below libraries for reproducibility\n",
    "#  !pip install catboost==0.23.2  ##### catboost classifier used\n",
    "# !pip install prince             ##### Multiple correspondence analysis used for categorical columns using this library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np\n",
    "import prince\n",
    "\n",
    "from catboost import CatBoostClassifier\n",
    "pd.options.mode.chained_assignment = None\n",
    "pd.set_option('display.max_columns', None)\n",
    "\n",
    "from sklearn.metrics import *\n",
    "from sklearn.model_selection import *\n",
    "from sklearn.decomposition import *\n",
    "\n",
    "import matplotlib\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Some custom functions to be used later for aggregation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_groupby(df,primary_keys,dictionary_ops,renaming_dict):\n",
    "    '''\n",
    "        primary_keys is a list of primary keys.\n",
    "        dictionary_ops is the dictionay having the operations to be performed (example :- {'location_number':'count'})\n",
    "        renaming_dict is the column to be renamed after joining and resetting index\n",
    "    '''\n",
    "    return df.groupby(primary_keys).agg(dictionary_ops).reset_index().rename(columns=renaming_dict)\n",
    "\n",
    "def data_left_join(df1,df2,primary_key):\n",
    "    '''\n",
    "        df1 :- First dataframe\n",
    "        df2 :- Second Dataframe\n",
    "        primary_key :- The list of primary keys on which one needs to left join\n",
    "    '''\n",
    "    return df1.merge(df2,how='left',on=primary_key)\n",
    "\n",
    "def updated_df(df,primary_key,operation,columns):\n",
    "    for cols in columns:\n",
    "        print('Aggregate ',operation ,' on column- ',cols)\n",
    "        df       = data_left_join(df,\n",
    "                                   my_groupby(df,\n",
    "                                              [primary_key],\n",
    "                                              {cols:operation},\n",
    "                                              {cols:primary_key+'_'+operation+'_'+cols}),\n",
    "                                   primary_key)\n",
    "\n",
    "    return df\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 1:- Loading train and test data \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_path                          = r'D:\\machine_hacks\\machine_hack_17\\Train.csv'\n",
    "test_path                           = r'D:\\machine_hacks\\machine_hack_17\\Test.csv'\n",
    "submis_path                         = r'D:\\machine_hacks\\machine_hack_17\\Sample_Submission.csv'\n",
    "\n",
    "train_data                          = pd.read_csv(train_path)\n",
    "test_data                           = pd.read_csv(test_path)\n",
    "train_y                             = train_data.Class.values\n",
    "train_data                          = train_data.drop(['Class'],axis=1)    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2:- Performing Multiple Correspondence Analysis for categorical features\n",
    "We will create new feature for train and test dataset that best explains the correspondence/association between categorical features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "concat_df                          = pd.concat((train_data,test_data),axis=0)\n",
    "cat_cols                           = ['Area_Code','Locality_Code','Region_Code','Species']\n",
    "mca                                = prince.MCA(n_components=1,random_state=202020).fit(concat_df[cat_cols])\n",
    "train_data.loc[:,'mca_cat1']        = mca.transform(train_data[cat_cols])[0]\n",
    "test_data.loc[:,'mca_cat1']         = mca.transform(test_data[cat_cols])[0]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3:- Principal Component Analysis on the numerical data given"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_cols                           = ['Height','Diameter']\n",
    "pca                                = PCA(n_components=1,random_state=202020).fit(concat_df[num_cols])\n",
    "train_data.loc[:,'pca_num']        = pca.transform(train_data[num_cols])[:,0]\n",
    "test_data.loc[:,'pca_num']         = pca.transform(test_data[num_cols])[:,0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 4:- Creation of EFB (Exclusive Feature Bundles)\n",
    "This is done to ensure that in the test data we have the combination of one categorical variable with other. This will make sure that if we have a novel value of a categorical data, we will be able to associate it with a categorical column that the data has already seen in train data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "del concat_df\n",
    "concat_df                          = pd.concat((train_data,test_data),axis=0)\n",
    "concat_df['EFB1']                  = concat_df['Locality_Code'].astype(str)+'_'+concat_df['Species'].astype(str)\n",
    "concat_df['EFB2']                  = concat_df['Locality_Code'].astype(str)+'_'+concat_df['Region_Code'].astype(str)\n",
    "concat_df['EFB3']                  = concat_df['Species'].astype(str)+'_'+concat_df['Region_Code'].astype(str)\n",
    "concat_df['EFB4']                  = concat_df['Area_Code'].astype(str)+'_'+concat_df['Region_Code'].astype(str)\n",
    "concat_df['EFB5']                  = concat_df['Area_Code'].astype(str)+'_'+concat_df['Locality_Code'].astype(str)\n",
    "concat_df['EFB6']                  = concat_df['Area_Code'].astype(str)+'_'+concat_df['Species'].astype(str)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 5:- Aggregate feature Creation\n",
    "We create aggregate features keeping various categorical columns as primary keys."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Area_Code\n",
      "Aggregate  mean  on column-  Height\n",
      "Aggregate  mean  on column-  Diameter\n",
      "Aggregate  mean  on column-  mca_cat1\n",
      "Aggregate  mean  on column-  pca_num\n",
      "Aggregate  mean  on column-  ratio_height_diam\n",
      "Aggregate  std  on column-  Height\n",
      "Aggregate  std  on column-  Diameter\n",
      "Aggregate  std  on column-  mca_cat1\n",
      "Aggregate  std  on column-  pca_num\n",
      "Aggregate  std  on column-  ratio_height_diam\n",
      "Aggregate  min  on column-  Height\n",
      "Aggregate  min  on column-  Diameter\n",
      "Aggregate  min  on column-  mca_cat1\n",
      "Aggregate  min  on column-  pca_num\n",
      "Aggregate  min  on column-  ratio_height_diam\n",
      "Aggregate  max  on column-  Height\n",
      "Aggregate  max  on column-  Diameter\n",
      "Aggregate  max  on column-  mca_cat1\n",
      "Aggregate  max  on column-  pca_num\n",
      "Aggregate  max  on column-  ratio_height_diam\n",
      "Aggregate  median  on column-  Height\n",
      "Aggregate  median  on column-  Diameter\n",
      "Aggregate  median  on column-  mca_cat1\n",
      "Aggregate  median  on column-  pca_num\n",
      "Aggregate  median  on column-  ratio_height_diam\n",
      "\n",
      "\n",
      "Locality_Code\n",
      "Aggregate  mean  on column-  Height\n",
      "Aggregate  mean  on column-  Diameter\n",
      "Aggregate  mean  on column-  mca_cat1\n",
      "Aggregate  mean  on column-  pca_num\n",
      "Aggregate  mean  on column-  ratio_height_diam\n",
      "Aggregate  std  on column-  Height\n",
      "Aggregate  std  on column-  Diameter\n",
      "Aggregate  std  on column-  mca_cat1\n",
      "Aggregate  std  on column-  pca_num\n",
      "Aggregate  std  on column-  ratio_height_diam\n",
      "Aggregate  min  on column-  Height\n",
      "Aggregate  min  on column-  Diameter\n",
      "Aggregate  min  on column-  mca_cat1\n",
      "Aggregate  min  on column-  pca_num\n",
      "Aggregate  min  on column-  ratio_height_diam\n",
      "Aggregate  max  on column-  Height\n",
      "Aggregate  max  on column-  Diameter\n",
      "Aggregate  max  on column-  mca_cat1\n",
      "Aggregate  max  on column-  pca_num\n",
      "Aggregate  max  on column-  ratio_height_diam\n",
      "Aggregate  median  on column-  Height\n",
      "Aggregate  median  on column-  Diameter\n",
      "Aggregate  median  on column-  mca_cat1\n",
      "Aggregate  median  on column-  pca_num\n",
      "Aggregate  median  on column-  ratio_height_diam\n",
      "\n",
      "\n",
      "Region_Code\n",
      "Aggregate  mean  on column-  Height\n",
      "Aggregate  mean  on column-  Diameter\n",
      "Aggregate  mean  on column-  mca_cat1\n",
      "Aggregate  mean  on column-  pca_num\n",
      "Aggregate  mean  on column-  ratio_height_diam\n",
      "Aggregate  std  on column-  Height\n",
      "Aggregate  std  on column-  Diameter\n",
      "Aggregate  std  on column-  mca_cat1\n",
      "Aggregate  std  on column-  pca_num\n",
      "Aggregate  std  on column-  ratio_height_diam\n",
      "Aggregate  min  on column-  Height\n",
      "Aggregate  min  on column-  Diameter\n",
      "Aggregate  min  on column-  mca_cat1\n",
      "Aggregate  min  on column-  pca_num\n",
      "Aggregate  min  on column-  ratio_height_diam\n",
      "Aggregate  max  on column-  Height\n",
      "Aggregate  max  on column-  Diameter\n",
      "Aggregate  max  on column-  mca_cat1\n",
      "Aggregate  max  on column-  pca_num\n",
      "Aggregate  max  on column-  ratio_height_diam\n",
      "Aggregate  median  on column-  Height\n",
      "Aggregate  median  on column-  Diameter\n",
      "Aggregate  median  on column-  mca_cat1\n",
      "Aggregate  median  on column-  pca_num\n",
      "Aggregate  median  on column-  ratio_height_diam\n",
      "\n",
      "\n",
      "Species\n",
      "Aggregate  mean  on column-  Height\n",
      "Aggregate  mean  on column-  Diameter\n",
      "Aggregate  mean  on column-  mca_cat1\n",
      "Aggregate  mean  on column-  pca_num\n",
      "Aggregate  mean  on column-  ratio_height_diam\n",
      "Aggregate  std  on column-  Height\n",
      "Aggregate  std  on column-  Diameter\n",
      "Aggregate  std  on column-  mca_cat1\n",
      "Aggregate  std  on column-  pca_num\n",
      "Aggregate  std  on column-  ratio_height_diam\n",
      "Aggregate  min  on column-  Height\n",
      "Aggregate  min  on column-  Diameter\n",
      "Aggregate  min  on column-  mca_cat1\n",
      "Aggregate  min  on column-  pca_num\n",
      "Aggregate  min  on column-  ratio_height_diam\n",
      "Aggregate  max  on column-  Height\n",
      "Aggregate  max  on column-  Diameter\n",
      "Aggregate  max  on column-  mca_cat1\n",
      "Aggregate  max  on column-  pca_num\n",
      "Aggregate  max  on column-  ratio_height_diam\n",
      "Aggregate  median  on column-  Height\n",
      "Aggregate  median  on column-  Diameter\n",
      "Aggregate  median  on column-  mca_cat1\n",
      "Aggregate  median  on column-  pca_num\n",
      "Aggregate  median  on column-  ratio_height_diam\n",
      "\n",
      "\n",
      "Aggregate  nunique  on column-  Species\n",
      "Aggregate  nunique  on column-  Species\n",
      "Aggregate  nunique  on column-  Species\n",
      "Aggregate  nunique  on column-  Locality_Code\n",
      "Aggregate  nunique  on column-  Locality_Code\n",
      "Aggregate  nunique  on column-  Locality_Code\n",
      "Aggregate  nunique  on column-  Region_Code\n",
      "Aggregate  nunique  on column-  Region_Code\n",
      "Aggregate  nunique  on column-  Region_Code\n"
     ]
    }
   ],
   "source": [
    "concat_df['ratio_height_diam']     = np.where(concat_df['Diameter']!=0,concat_df['Height']/concat_df['Diameter'],np.NAN)\n",
    "aggregation_columns                = ['Height','Diameter','mca_cat1','pca_num','ratio_height_diam']\n",
    "numerical_aggregation_primary_keys = ['Area_Code','Locality_Code','Region_Code','Species']\n",
    "\n",
    "for cols in numerical_aggregation_primary_keys:\n",
    "    print(cols)\n",
    "    concat_df                       = updated_df(concat_df,cols,'mean',aggregation_columns)\n",
    "    concat_df                       = updated_df(concat_df,cols,'std',aggregation_columns)\n",
    "    concat_df                       = updated_df(concat_df,cols,'min',aggregation_columns)\n",
    "    concat_df                       = updated_df(concat_df,cols,'max',aggregation_columns)\n",
    "    concat_df                       = updated_df(concat_df,cols,'median',aggregation_columns)\n",
    "    print('\\n')\n",
    "\n",
    "concat_df                          = updated_df(concat_df,'Area_Code','nunique',['Species'])\n",
    "concat_df                          = updated_df(concat_df,'Locality_Code','nunique',['Species'])\n",
    "concat_df                          = updated_df(concat_df,'Region_Code','nunique',['Species'])\n",
    "\n",
    "concat_df                          = updated_df(concat_df,'Area_Code','nunique',['Locality_Code'])\n",
    "concat_df                          = updated_df(concat_df,'Region_Code','nunique',['Locality_Code'])\n",
    "concat_df                          = updated_df(concat_df,'Species','nunique',['Locality_Code'])\n",
    "\n",
    "concat_df                          = updated_df(concat_df,'Area_Code','nunique',['Region_Code'])\n",
    "concat_df                          = updated_df(concat_df,'Locality_Code','nunique',['Region_Code'])\n",
    "concat_df                          = updated_df(concat_df,'Species','nunique',['Region_Code'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 6:- Creating final train and test dataset "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "testcount                          = len(test_data)\n",
    "count                              = len(concat_df)-testcount\n",
    "\n",
    "train_data                         = concat_df[:count]\n",
    "test_data                          = concat_df[count:]\n",
    "\n",
    "##### We identify categorical columns here\n",
    "cat_cols                           = ['Area_Code','Locality_Code','Region_Code','Species','EFB1','EFB2','EFB3','EFB4','EFB5','EFB6']\n",
    "for cols in cat_cols:\n",
    "    train_data[cols]               = train_data[cols].astype(str)\n",
    "    test_data[cols]                = test_data[cols].astype(str)\n",
    "    \n",
    "train                              = train_data.values\n",
    "test                               = test_data.values\n",
    "cate_features_index                = np.where(train_data.dtypes == object)[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 7:- Model training and validation\n",
    "Make sure the colab session has GPU selected as Hardware accelerator in the runtime type."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validation logloss model 1 fold- 1 :  0.7570568117337814\n",
      "validation logloss model 2 fold- 1 :  0.77118285261914\n",
      "validation logloss fold- 1 :  0.7570561708009502\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 2 :  0.7199792111600761\n",
      "validation logloss model 2 fold- 2 :  0.73690868489152\n",
      "validation logloss fold- 2 :  0.7199792132847787\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 3 :  0.6349443709643815\n",
      "validation logloss model 2 fold- 3 :  0.6270544694185234\n",
      "validation logloss fold- 3 :  0.6270326096270531\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 4 :  0.6960311012853535\n",
      "validation logloss model 2 fold- 4 :  0.689148731580448\n",
      "validation logloss fold- 4 :  0.6890668734053383\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 5 :  0.7244444068784825\n",
      "validation logloss model 2 fold- 5 :  0.7166703143539669\n",
      "validation logloss fold- 5 :  0.7165812258396086\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 6 :  0.7387854767145599\n",
      "validation logloss model 2 fold- 6 :  0.7332455167785642\n",
      "validation logloss fold- 6 :  0.7327684180822011\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 7 :  0.6071948758209431\n",
      "validation logloss model 2 fold- 7 :  0.6294865837747621\n",
      "validation logloss fold- 7 :  0.6071948758295744\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 8 :  0.6882451740087407\n",
      "validation logloss model 2 fold- 8 :  0.684560746744721\n",
      "validation logloss fold- 8 :  0.6834445594753922\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 9 :  0.7455297568540731\n",
      "validation logloss model 2 fold- 9 :  0.7138093845200117\n",
      "validation logloss fold- 9 :  0.7138093845165171\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 10 :  0.663453007755255\n",
      "validation logloss model 2 fold- 10 :  0.6801218690280959\n",
      "validation logloss fold- 10 :  0.6634529931222735\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 11 :  0.7410961015314432\n",
      "validation logloss model 2 fold- 11 :  0.7412785469091688\n",
      "validation logloss fold- 11 :  0.7368969579569344\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 12 :  0.6482127687256167\n",
      "validation logloss model 2 fold- 12 :  0.6644873103024792\n",
      "validation logloss fold- 12 :  0.6482127300951899\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 13 :  0.664957282847837\n",
      "validation logloss model 2 fold- 13 :  0.6620628081667621\n",
      "validation logloss fold- 13 :  0.6599833775739852\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 14 :  0.8027618882679344\n",
      "validation logloss model 2 fold- 14 :  0.8210356984664781\n",
      "validation logloss fold- 14 :  0.8027618236515592\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 15 :  0.7230563374478253\n",
      "validation logloss model 2 fold- 15 :  0.7368064103103031\n",
      "validation logloss fold- 15 :  0.7230554682099917\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 16 :  0.7056788232134416\n",
      "validation logloss model 2 fold- 16 :  0.7051026953668114\n",
      "validation logloss fold- 16 :  0.6996700716129385\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 17 :  0.7785510039335847\n",
      "validation logloss model 2 fold- 17 :  0.7877787057740506\n",
      "validation logloss fold- 17 :  0.7785204233610633\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 18 :  0.7397948011227469\n",
      "validation logloss model 2 fold- 18 :  0.7416398327826901\n",
      "validation logloss fold- 18 :  0.7367205136225037\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 19 :  0.6923067760690469\n",
      "validation logloss model 2 fold- 19 :  0.6913495269549135\n",
      "validation logloss fold- 19 :  0.6876076800623707\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 20 :  0.6420642705950225\n",
      "validation logloss model 2 fold- 20 :  0.6384548876957259\n",
      "validation logloss fold- 20 :  0.637490337848874\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 21 :  0.6577596055844699\n",
      "validation logloss model 2 fold- 21 :  0.6769665304343508\n",
      "validation logloss fold- 21 :  0.6577596001492496\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 22 :  0.6987883564543248\n",
      "validation logloss model 2 fold- 22 :  0.7037490442806479\n",
      "validation logloss fold- 22 :  0.6982412355469106\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 23 :  0.7634466638549522\n",
      "validation logloss model 2 fold- 23 :  0.7693613684131146\n",
      "validation logloss fold- 23 :  0.763064660925316\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 24 :  0.6714806206767882\n",
      "validation logloss model 2 fold- 24 :  0.6650762972384937\n",
      "validation logloss fold- 24 :  0.6649357228260437\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 25 :  0.8150852481082745\n",
      "validation logloss model 2 fold- 25 :  0.8125311109921137\n",
      "validation logloss fold- 25 :  0.8104248836916436\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 26 :  0.6965622468199767\n",
      "validation logloss model 2 fold- 26 :  0.6893805329303035\n",
      "validation logloss fold- 26 :  0.6892136203315451\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 27 :  0.7211855430482866\n",
      "validation logloss model 2 fold- 27 :  0.7238971262776169\n",
      "validation logloss fold- 27 :  0.7189704573193851\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 28 :  0.7923511477722732\n",
      "validation logloss model 2 fold- 28 :  0.8049191430866531\n",
      "validation logloss fold- 28 :  0.7923467449402641\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 29 :  0.7693052410737705\n",
      "validation logloss model 2 fold- 29 :  0.7604977650875914\n",
      "validation logloss fold- 29 :  0.7604453665577977\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 30 :  0.6904612747036654\n",
      "validation logloss model 2 fold- 30 :  0.6673488852196046\n",
      "validation logloss fold- 30 :  0.6673488849189204\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 31 :  0.6876811149891369\n",
      "validation logloss model 2 fold- 31 :  0.6830633338125058\n",
      "validation logloss fold- 31 :  0.6822972561619224\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 32 :  0.7455599863149333\n",
      "validation logloss model 2 fold- 32 :  0.7513771393852132\n",
      "validation logloss fold- 32 :  0.7452338086553775\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 33 :  0.7552713064668323\n",
      "validation logloss model 2 fold- 33 :  0.763944951713345\n",
      "validation logloss fold- 33 :  0.7552431016565821\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 34 :  0.6315678030516582\n",
      "validation logloss model 2 fold- 34 :  0.6337884043090254\n",
      "validation logloss fold- 34 :  0.6284289797278169\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 35 :  0.7220809890414263\n",
      "validation logloss model 2 fold- 35 :  0.7226599801532001\n",
      "validation logloss fold- 35 :  0.7179264301996163\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 36 :  0.6769483486025974\n",
      "validation logloss model 2 fold- 36 :  0.6908419898245151\n",
      "validation logloss fold- 36 :  0.6769483800341591\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 37 :  0.6503592022917943\n",
      "validation logloss model 2 fold- 37 :  0.657459420916618\n",
      "validation logloss fold- 37 :  0.650259938937348\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 38 :  0.7634126557609826\n",
      "validation logloss model 2 fold- 38 :  0.750161864770748\n",
      "validation logloss fold- 38 :  0.7501584011334987\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 39 :  0.7527741519717981\n",
      "validation logloss model 2 fold- 39 :  0.7497408743666754\n",
      "validation logloss fold- 39 :  0.7484950996528702\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 40 :  0.7295309121514328\n",
      "validation logloss model 2 fold- 40 :  0.744907901465272\n",
      "validation logloss fold- 40 :  0.7295299407052434\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 41 :  0.7843017322018573\n",
      "validation logloss model 2 fold- 41 :  0.7795945920768355\n",
      "validation logloss fold- 41 :  0.7781665948328677\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 42 :  0.6591320559699554\n",
      "validation logloss model 2 fold- 42 :  0.6930027389686426\n",
      "validation logloss fold- 42 :  0.6591320559701865\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 43 :  0.6571689097701863\n",
      "validation logloss model 2 fold- 43 :  0.6716988375899121\n",
      "validation logloss fold- 43 :  0.657168640837869\n",
      "\n",
      "\n",
      "validation logloss model 1 fold- 44 :  0.7450649347010498\n",
      "validation logloss model 2 fold- 44 :  0.7541522681801966\n",
      "validation logloss fold- 44 :  0.7450501856862464\n",
      "\n",
      "\n",
      "OOF logloss:-  0.7083604941499381\n"
     ]
    }
   ],
   "source": [
    "oof_pred               = np.zeros((len(train),8))\n",
    "y_pred_final           = np.zeros((len(test), 8))\n",
    "num_models             = 2\n",
    "\n",
    "n_splits               = 44\n",
    "error                  = []\n",
    "kf                     = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "\n",
    "for fold, (tr_ind, val_ind) in enumerate(kf.split(train, train_y)):\n",
    "    wghts              = [0]*num_models\n",
    "    logloss            = []\n",
    "    \n",
    "    X_train, X_val     = train[tr_ind], train[val_ind]\n",
    "    y_train, y_val     = train_y[tr_ind], train_y[val_ind]\n",
    "    \n",
    "    \n",
    "    \n",
    "    model1             = CatBoostClassifier(n_estimators=1200,random_state=202020,verbose=False,task_type='GPU')\n",
    "    model1.fit(X_train,y_train,cat_features = cate_features_index,eval_set=(X_val,y_val))\n",
    "    val_pred1          = model1.predict_proba(X_val)\n",
    "    logloss.append(log_loss(y_val,val_pred1))\n",
    "    print('validation logloss model 1 fold-',fold+1,': ',log_loss(y_val,val_pred1))\n",
    "    \n",
    "    \n",
    "    model2             = CatBoostClassifier(n_estimators=1000,random_state=202020,verbose=False,task_type='GPU')\n",
    "    model2.fit(X_train,y_train,cat_features = cate_features_index,eval_set=(X_val,y_val))\n",
    "    val_pred2          = model2.predict_proba(X_val)\n",
    "    logloss.append(log_loss(y_val,val_pred2))\n",
    "    print('validation logloss model 2 fold-',fold+1,': ',log_loss(y_val,val_pred2))\n",
    "    \n",
    "    \n",
    "    wghts              = np.exp(-1000*np.array(logloss/sum(logloss)))\n",
    "    wghts              = wghts/sum(wghts)\n",
    "    \n",
    "    val_pred           = wghts[0]*val_pred1+wghts[1]*val_pred2\n",
    "    print('validation logloss fold-',fold+1,': ',log_loss(y_val, val_pred))\n",
    "    \n",
    "    oof_pred[val_ind]  = val_pred\n",
    "    \n",
    "    y_pred_final += (wghts[0]*model1.predict_proba(test)+wghts[1]*model2.predict_proba(test))/(n_splits)\n",
    "    \n",
    "    print('\\n')\n",
    "    \n",
    "print('OOF logloss:- ',(log_loss(train_y,oof_pred)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Final Step :- Output Creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission_df       = pd.read_csv(submis_path)\n",
    "columns_name        = submission_df.columns.tolist()\n",
    "output_df           = pd.DataFrame(y_pred_final,columns=columns_name)\n",
    "output_df.to_csv('output_trial_catboost_44folds_2models_gpu.csv',index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
